import sys, gzip, json, os
from collections import Counter


def calculate_TFIDF(TFs,IDFs):
	RES = Counter()
	for word in TFs:
		RES[word] = TFs[word] * IDFs[word]
	return RES

if __name__ == "__main__":
	print "reading idf"
	outfile = gzip.open("IDFmatrix.json.gz",'r')
	IDF = Counter(json.loads(outfile.readline()))
	outfile.close()
	print "removing invalid words"
	for k in [w for w in IDF if not all([p.isalpha() for p in w.split("-")])]:
		IDF[k] = 0
	IDF += Counter()
	print "detecting documents to be calculated"
	all_files = os.listdir("TFMATRIX")
	all_done = os.listdir("TFIDFs")
	print "remaining documents to calculate ", len(all_files) - len(all_done)
	iterator = 0
	for w in all_done:
		all_files.remove(w)
	for TFfile in all_files:
			iterator += 1
			tffile = gzip.open("TFMATRIX/"+TFfile,'r')
			TF = Counter(json.loads(tffile.readline()))
			tffile.close()
			tfidfs = gzip.open("TFIDFs/"+TFfile,'w')
			tfidfs.write(json.dumps(calculate_TFIDF(TF,IDF))+"\n")
			tfidfs.close()
			print "Calculated ", iterator, " documents"
	sys.exit()


